{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Attention Flow - Graph Tools.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPoLI0EjodXj+Hk09oFa4Vn",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "765e8d70e824428fb1e77a69f9fa3d94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_985bb13e2517490ebc64fac5c12a82a9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_79aa70a265b34e73b6988b00e8762761",
              "IPY_MODEL_3f4f0c0796a4436d96d0ea6561f8420f"
            ]
          }
        },
        "985bb13e2517490ebc64fac5c12a82a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "79aa70a265b34e73b6988b00e8762761": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_459f88e73da64af9825bfb4f8a40a824",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_32b4ffdc797244baaed5854196a474dc"
          }
        },
        "3f4f0c0796a4436d96d0ea6561f8420f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4af03e6925b944b1bf2c111acc4329a2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [01:23&lt;00:00, 2.78kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d27b0a0dd5f542758aad5ddfb96c3123"
          }
        },
        "459f88e73da64af9825bfb4f8a40a824": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "32b4ffdc797244baaed5854196a474dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4af03e6925b944b1bf2c111acc4329a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d27b0a0dd5f542758aad5ddfb96c3123": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ea6aa87398544e77872e25d5fb0b00db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_20777c90d6a84b62b0204c85a88157d6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c4e7d9e612594523ac8d5fa549fb842b",
              "IPY_MODEL_20f27a5940014e8c91abe0638a1a31b5"
            ]
          }
        },
        "20777c90d6a84b62b0204c85a88157d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c4e7d9e612594523ac8d5fa549fb842b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3e47b4f2fa3449ad97a75b15f5855190",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 434,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 434,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_09361de0600d440d9e5dd074beeaaf38"
          }
        },
        "20f27a5940014e8c91abe0638a1a31b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_65b8e29b67254b38afe9d7f514e71126",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 434/434 [00:01&lt;00:00, 375B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_afc6ed1215f942bda6bc76aad19d73bb"
          }
        },
        "3e47b4f2fa3449ad97a75b15f5855190": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "09361de0600d440d9e5dd074beeaaf38": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "65b8e29b67254b38afe9d7f514e71126": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "afc6ed1215f942bda6bc76aad19d73bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a3c5f4c440944e6afc58a0a9d65289e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_02fbf58c0a9e4cfbb96006e2c9b8fafc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_93fac4edf549475280930d1c2d4da312",
              "IPY_MODEL_e7509e4246034755b7bdcd03dc8100e1"
            ]
          }
        },
        "02fbf58c0a9e4cfbb96006e2c9b8fafc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "93fac4edf549475280930d1c2d4da312": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_88c6747291ed4d629784d1f5609ee48b",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1344997306,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1344997306,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ce35f568c5b0480a885d1534078dc6f4"
          }
        },
        "e7509e4246034755b7bdcd03dc8100e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_990bfe4f94c041cd85e97f3aafc138d8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.34G/1.34G [00:23&lt;00:00, 57.8MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d31e30ed1ddb4791a8deac870158847c"
          }
        },
        "88c6747291ed4d629784d1f5609ee48b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ce35f568c5b0480a885d1534078dc6f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "990bfe4f94c041cd85e97f3aafc138d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d31e30ed1ddb4791a8deac870158847c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jalammar/jalammar.github.io/blob/master/notebooks/Attention_Flow_Graph_Tools.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gVk2-ESTxV8y"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e_t6a85Vxja1"
      },
      "source": [
        "# Graph Tool Installation\n",
        "# https://colab.research.google.com/github/count0/colab-gt/blob/master/colab-gt.ipynb#scrollTo=GQ18Kd5F3uKe\n",
        "!echo \"deb http://downloads.skewed.de/apt bionic main\" >> /etc/apt/sources.list\n",
        "!apt-key adv --keyserver keys.openpgp.org --recv-key 612DEFB798507F25\n",
        "!apt-get update\n",
        "!apt-get install python3-graph-tool python3-cairo python3-matplotlib"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OjnMsOFbxeq6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237,
          "referenced_widgets": [
            "765e8d70e824428fb1e77a69f9fa3d94",
            "985bb13e2517490ebc64fac5c12a82a9",
            "79aa70a265b34e73b6988b00e8762761",
            "3f4f0c0796a4436d96d0ea6561f8420f",
            "459f88e73da64af9825bfb4f8a40a824",
            "32b4ffdc797244baaed5854196a474dc",
            "4af03e6925b944b1bf2c111acc4329a2",
            "d27b0a0dd5f542758aad5ddfb96c3123",
            "ea6aa87398544e77872e25d5fb0b00db",
            "20777c90d6a84b62b0204c85a88157d6",
            "c4e7d9e612594523ac8d5fa549fb842b",
            "20f27a5940014e8c91abe0638a1a31b5",
            "3e47b4f2fa3449ad97a75b15f5855190",
            "09361de0600d440d9e5dd074beeaaf38",
            "65b8e29b67254b38afe9d7f514e71126",
            "afc6ed1215f942bda6bc76aad19d73bb",
            "1a3c5f4c440944e6afc58a0a9d65289e",
            "02fbf58c0a9e4cfbb96006e2c9b8fafc",
            "93fac4edf549475280930d1c2d4da312",
            "e7509e4246034755b7bdcd03dc8100e1",
            "88c6747291ed4d629784d1f5609ee48b",
            "ce35f568c5b0480a885d1534078dc6f4",
            "990bfe4f94c041cd85e97f3aafc138d8",
            "d31e30ed1ddb4791a8deac870158847c"
          ]
        },
        "outputId": "5eb84e6d-2993-41ff-ca63-55b0f5efd4ef"
      },
      "source": [
        "\n",
        "model_name = 'bert-large-uncased'\n",
        "from transformers import BertTokenizer, BertForMaskedLM, DistilBertTokenizer, DistilBertModel\n",
        "tokenizer = BertTokenizer.from_pretrained(model_name)\n",
        "model = BertForMaskedLM.from_pretrained(model_name,\n",
        "                                  output_hidden_states=True,\n",
        "                                  output_attentions=True)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "765e8d70e824428fb1e77a69f9fa3d94",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ea6aa87398544e77872e25d5fb0b00db",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=434.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1a3c5f4c440944e6afc58a0a9d65289e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1344997306.0, style=ProgressStyle(descr…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-large-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LmN4BH1xxfwl"
      },
      "source": [
        "import graph_tool as gt\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "\n",
        "\n",
        "def get_adjmat(mat, input_tokens):\n",
        "    n_layers, length, _ = mat.shape\n",
        "    adj_mat = np.zeros(((n_layers+1)*length, (n_layers+1)*length))\n",
        "    labels_to_index = {}\n",
        "    for k in np.arange(length):\n",
        "        labels_to_index[str(k)+\"_\"+input_tokens[k]] = k\n",
        "\n",
        "    for i in np.arange(1,n_layers+1):\n",
        "        for k_f in np.arange(length):\n",
        "            index_from = (i)*length+k_f\n",
        "            label = \"L\"+str(i)+\"_\"+str(k_f)\n",
        "            labels_to_index[label] = index_from\n",
        "            for k_t in np.arange(length):\n",
        "                index_to = (i-1)*length+k_t\n",
        "                adj_mat[index_from][index_to] = mat[i-1][k_f][k_t]\n",
        "                \n",
        "    return adj_mat, labels_to_index \n",
        "\n",
        "\n",
        "def convert_adjmat_tomats(adjmat, n_layers, l):\n",
        "   mats = np.zeros((n_layers,l,l))\n",
        "   \n",
        "   for i in np.arange(n_layers):\n",
        "       mats[i] = adjmat[(i+1)*l:(i+2)*l,i*l:(i+1)*l]\n",
        "       \n",
        "   return mats"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k7t8wKQgx0jd"
      },
      "source": [
        "sentence = \"The author talked to Sara about [MASK] book\"\n",
        "encoded_input = tokenizer(sentence, return_tensors='pt')\n",
        "# # of tokens: 10 tokens\n",
        "\n",
        "all_hidden_states, all_attentions = model(**encoded_input)[-2:]\n",
        "\n",
        "tokens = ['[cls]']+tokenizer.tokenize(sentence)+['[sep]']\n",
        "\n",
        "# all_attention is a list of tensors, one per layer.\n",
        "# e.g. Each is (1, 12, 10, 10) which is (batch_size, attention heads, tokens in, tokens out)\n",
        "_attentions = [att.detach().numpy() for att in all_attentions]\n",
        "\n",
        "# attentions_mat is cleaner to deal with\n",
        "# (6, 12, 10, 10) which is (layer, attention head, tokens in, tokens out)\n",
        "attentions_mat = np.asarray(_attentions)[:,0]\n"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ltmjLohAx3yR"
      },
      "source": [
        "# Sum the attention values of all the heads, then normalize by dividing by the \n",
        "# number of heads\n",
        "res_att_mat = attentions_mat.sum(axis=1)/attentions_mat.shape[1]\n",
        "\n",
        "# Add 1 to each token's attention value to itself.\n",
        "# This accounts for the residual connection adding the token'the position's\n",
        "# input back after self-attention\n",
        "# [None,...] creates\n",
        "res_att_mat = res_att_mat + np.eye(res_att_mat.shape[1])[None,...]\n",
        "\n",
        "\n",
        "# Normalize so that all the attention going into a node adds up to one?\n",
        "res_att_mat = res_att_mat / res_att_mat.sum(axis=-1)[...,None]"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0nCJsM4Aydgo"
      },
      "source": [
        "# We want to end up creating a weighted graph. Turning the attention matrix\n",
        "# into an adjacency matrix is a step in that direction\n",
        "res_adj_mat, res_labels_to_index = get_adjmat(mat=res_att_mat, input_tokens=tokens)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XTC9aVXCyLUH"
      },
      "source": [
        "# Create the attention graph\n",
        "g = gt.Graph()\n",
        "# Set which nodes are connected to which others nodes using tha adjacency matrix\n",
        "# (No weights/capacities set yet)\n",
        "indices = res_adj_mat.nonzero()\n",
        "g.add_edge_list(np.transpose(indices))\n",
        "\n",
        "# Set the capacities (attention weights)\n",
        "edge_capacities = g.new_edge_property(\"double\")\n",
        "edge_capacities.a = res_adj_mat [indices]\n",
        "g.ep['capacity'] = edge_capacities"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xz5V3tkByvHP"
      },
      "source": [
        "\n",
        "output_nodes = []\n",
        "input_nodes = []\n",
        "for key in res_labels_to_index:\n",
        "    if 'L24' in key:\n",
        "        output_nodes.append(key)\n",
        "    if res_labels_to_index[key] < attentions_mat.shape[-1]:\n",
        "        input_nodes.append(key)\n"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ChBC0MyuyTQd"
      },
      "source": [
        "\n",
        "from graph_tool.flow import edmonds_karp_max_flow\n",
        "\n",
        "labels_to_index = res_labels_to_index\n",
        "length=attentions_mat.shape[-1]\n",
        "number_of_nodes = len(labels_to_index)\n",
        "flow_values_gt_2=np.zeros((number_of_nodes, len(input_nodes)))\n",
        "cap = g.edge_properties[\"capacity\"]\n",
        "\n",
        "# For each position in each layer\n",
        "for key in labels_to_index:\n",
        "    # skip input nodes (attention weights)\n",
        "    if key not in input_nodes:\n",
        "        u = labels_to_index[key] # The node we'll trace (source)\n",
        "        src = g.vertex(u)\n",
        "        # For each input token\n",
        "        for inp_node_key in input_nodes:\n",
        "            v = labels_to_index[inp_node_key] # The input token (target)\n",
        "            tgt = g.vertex(v)\n",
        "\n",
        "            # Calculate the maximum flow\n",
        "            # https://graph-tool.skewed.de/static/doc/flow.html#graph_tool.flow.edmonds_karp_max_flow\n",
        "            res = edmonds_karp_max_flow(g, src, tgt, cap)\n",
        "            res.a = cap.a - res.a  # the actual flow\n",
        "            max_flow = sum(res[e] for e in tgt.in_edges())\n",
        "            flow_values_gt_2[u][labels_to_index[inp_node_key]] = max_flow\n",
        "\n",
        "        # Normalize the attention going into the node so it adds up to one    \n",
        "        flow_values_gt_2[u] /= flow_values_gt_2[u].sum()"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZUPcL_6uz85H",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "59e78033-36ea-40d4-808e-1b9acbd94781"
      },
      "source": [
        "# Attentions are now here:\n",
        "attention_flow = flow_values_gt_2.reshape((-1, flow_values_gt_2.shape[1], flow_values_gt_2.shape[1]))\n",
        "print(attention_flow.shape) # Dimensions: Layers, tokens in, tokens out"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(25, 10, 10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RFFXwI_1s98E"
      },
      "source": [
        "import itertools\n",
        "import matplotlib.pyplot as plt\n",
        "def plot_attention_heatmap(att, s_position, t_positions, sentence):\n",
        "\n",
        "  cls_att = np.flip(att[:,s_position, t_positions], axis=0)\n",
        "  xticklb = input_tokens= list(itertools.compress(['<cls>']+sentence.split(), [i in t_positions for i in np.arange(len(sentence)+1)]))\n",
        "  yticklb = [str(i) if i%2 ==0 else '' for i in np.arange(att.shape[0],0, -1)]\n",
        "  ax = sns.heatmap(cls_att, xticklabels=xticklb, yticklabels=yticklb, cmap=\"YlOrRd\")\n",
        "  return ax"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FolxCv-MzeWd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        },
        "outputId": "9a7a2979-8928-4b09-a270-3797ddeaa2f8"
      },
      "source": [
        "\n",
        "import seaborn as sns\n",
        "src = 7\n",
        "targets = (2,5) \n",
        "plt.figure(1,figsize=(3,6))\n",
        "plot_attention_heatmap(attention_flow, src, targets, sentence)\n",
        "# (attentions_mat.sum(axis=1)/attentions_mat.shape[1], src, t_positions=targets, sentence=sentence)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f0c1e2caba8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAANsAAAFlCAYAAABiNJ3xAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAZ5klEQVR4nO3df7BfdX3n8efrJhLA1EBCYdsESZQoG4tjJUXsVPkRxdgqwTFZI+0kOJTU7iLOrBXTcYpAqzWtmHVc6BgXAsWZJRi79FJTWDSNtq5CAqIQIPY2iLlxoQuJsZFCDLz3j3OufPnm3u/53vv9nnM/95zXY+ZMzvf8+J7PuZP39/PjfL7vryICMyvfwGQXwKwpHGxmFXGwmVXEwWZWEQebWUUcbGYVmV72BT6j1/rZQu63jzs02UVIxqL9j6nT/qv78P/mE7Gr4zWqVnqwmU1EHZtcdbwnsyS5ZrMk1bEWcLBZkhxsZhWpY7CVck+S1kjaIWnHd/hJGZcwm3JKCbaI2BARiyNi8VkcV8YlrOYG+rCkxs1IS1JSD8j6xMFmSUqxZuqVg82SVMdgq+M9mSXJNZslqY61gIPNkuRgM6uIg82sInUMtjrek1mSxl2zSToxIv61jMKYjahjLdAx2CTNbt8E3Cvp1wFFxL7SSmaN1rhgA54CHm/bNhe4HwjgVaOdJGkNsAbgvZyI50faeNUx2Iru6aPALuCCiFgQEQuA4Xx91EADT0Q2G03Hmi0irpW0CVgvaQ/wCbIazaxUdazZCgdIImIYWCHpAuBu4NjSS2WN18hgk3QaWT9tK1mwvTrfvjQi7iy3eNZUdfyKTccPEEmXA38LfAh4CDg/Ih7Kd3+q5LJZg9Xxy6NFZboUOCMiLgTOAf5E0ofzfXX88LEGkbRU0i5JQ5LWjrL/rZLul3RY0vK2fasl/XO+rO7mekXNyIGIOAgQET+UdA6wWdIpONisRGXXTJKmAdcBbweGge2SBiPi4ZbDfgRcDPxR27mzyQYLF5MNGN6Xn7u/0zWL7ulJSW8YeZEH3ruAE4DTu7kps4mooBl5JjAUEbsj4hBwK7Cs9YCI+GFEfB94oe3cdwB3R8S+PMDuBpZ2c0+drAKeaCvA4YhYBby16M3NJqofwdaa5S1f1rRcYi6wp+X1cL6tGxM6t+g523CHfd/qsmBm49aPZmREbAA29OGt+iLFQRuzKuwFTm55PS/fVtq5DjZLUgV9tu3AQkkLJB0FrAQGuyzeXcD5ko6XdDxwfr6t8J76zhmRrVdlB1tEHAYuIwuSR4DbImKnpGvy2VJI+g1Jw8AK4AuSdubn7gP+lCxgtwPXdPMNGEWUO9XRP4b4Iv8Y4ouKfgzxjj78v3l3Yj+G6GakWUWcg8SSVMdawMFmSXKwmVUkqc5WnzjYLEl1rNnqeE9mSXLNZkmqYy3gYLMkOdjMKqIajpA42CxJA6rfxKM61tZmSSpK+PMKSX8u6RZJF7Xtu77DeZ6IbD2Rel9SU1SzbSR7vvgVYKWkr0iake87a6yTnBHZeqU+LKkp6rO9OiLem6/fLunjwNaRryCYlUU17LMVBdsMSQMR8QJARHxS0l7gm8DM0ktnViNFzcg7gPNaN0TETcBHAH85y0rTuD5bRFwBDEtaImlmy/Y7gcvLLpw1V+OCTdKHaEk/Lqk1r94nyyyYNduAouclNUV9tjVk6ccPSppPlg15fkR8jjQHfKwm6vify+nHzSri9OOWpDr22YpqtlXA4dYNeQqwVZK+UFqprPFSDJZeOf24JamJD7XNJsVADWs2Z0Q2q0gpweaJyNarJg6QmE0K4T6bWSVSrJl65W9qm1XENZslqY41m4PNkpTiROJeOdgsSa7ZzCpSw1jzAIlZVVyzWZI8N9KsIu6zmVWkcRORJc2S9GlJj0raJ+lpSY/k28ac9OiJyNYrKXpeUlM0QHIbsB84JyJmR8Qc4Nx8221jneSJyGZHKgq2+RGxLiKeGNkQEU9ExDrglHKLZk1Wx/TjRcH2uKQrJJ00skHSSZI+Buwpt2jWZHX8ik1RsL0PmAN8Q9J+SfuAbcBs4D+VXDZrsDr22YpykOyXtBG4G/jOSFo7AElLgTtLLp9ZbRSNRl5OlhH5Mo7MiPypMgtmzTag3pfUFD1nuxRnRLZJkGKfq1fOiGxJqmOwOSOyJUlEz0tqioJtFfBE64aIOBwRq4C3llYqsxpyRmRLUh2bkZ6IbElSisOJPXJGZEuSBnpfCq8hLZW0S9KQpLWj7J8haVO+/558RB5JL5N0s6QH84n5f9zNPTkjsjWSpGnAdcA7gUXA+yUtajvsEmB/RJwKrAfW5dtXADMi4nTgDOAPRgKxE6dFsCRVMDfyTGAoInZHxCHgVmBZ2zHLgJvz9c3AEkkCAni5pOnAMcAh4KdFF3SwWZrKn0Iyl5dOph/Ot416TP67hAfI5gpvBn4G/F/gR8BnImJf0QU9QGJJ6qbPVfge0hqy34UfsSEiNvT+zpwJPA/8KnA88I+SvhYRuzud5GCzJKkPY/95YI0VXHuBk1tez8u3jXbMcN5knAU8DVwE3BkRPwf+VdK3gMVAx2BzM9KaajuwUNICSUcBK4HBtmMGgdX5+nJga0QEWdPxPABJLwfOAh4tuqBrNktSP5qRnUTEYUmXAXcB04AbI2KnpGuAHRExCNwA3CJpCNhHFpCQjWJulLSTbI7wxoj4ftE1HWyWpgqmkETEFmBL27YrW9afJRvmbz/v4GjbizjYLEll12yTwcFmSfJ0LTObMNdslqQ6zvr3RGRLUhUTkatWlPBnsaR/kPQlSSdLulvSAUnbJf36WOd5IrL1rIYZf4ri/3rgL4CvAv8H+EJEzALW5vvMrEtFwfayiPj7iPifQETEZrKVrwNHl146a6w6ZkQuGiB5VtL5ZHPCQtKFEXG7pLPJJmKalaKOQ/9FwfZBsmbkC8A7gD+UdBPZBM1Lyy2aNVmKAxy9Kkr48z2yIBvx4XxB0gfI+nFmfdePWf+p6eXz4+q+lcKsATrWbJLGmsks4KQx9pn1rmnNSLKAegfZL422Em5CWolq2IosDLa/A2ZGxAPtOyRtK6VEZjRwNDIiLumw76L+F8csU8fRyBreklmaPOvf0lTDTptn/VuSGjfrf6I86996pQH1vKQmwfg3qyf32SxJNeyyOdgsTSk2A3vlYLM01S/WHGyWphRHE3tVw1syS5NrNkuS+2xmFfFopFlFXLOZVaWGownjviVJc8ooiFndFWVE/rSkE/L1xZJ2A/dIejxPZzfWeZ6IbL1pYEbk34mIp/L1vwTeFxGnAm8Hrh3rJE9Etp4N9GFJTFGfbbqk6RFxGDgmIrYDRMQPJM0ov3jWWAnWTL3qJtf/FknnAXdK+pyksyVdDRyRl8TMxlaUg+Tzkh4E/hB4TX78QuB24E/LL541VoLNwF4VDv1HxDZgW/v2PCPyxv4XyYxGNiM7cUZkK08NRyOdEdnS1MBmpDMim/WJMyJbmhJsBvbKGZEtTQ1sRppNjqbVbGaTpn6x5ozIZlVxRmRLU9Oes5lNmgSDpVcONktTDUcja3hLZmlyzWZpcjPSrBrOiGxWlQpGIyUtlbRL0pCktaPsnyFpU77/HknzW/a9XtK3Je2U9KCkowtvaZx/ArNqlJyDRNI04DrgncAi4P2SFrUddgmwP8+7sx5Yl587HfgS8MGIeB1wDvDzbm7JrInOBIYiYndEHAJuBZa1HbMMuDlf3wwskSTgfOD7EfE9gIh4OiKeL7qgg83S1IdmZOtMpnxZ03KFucCeltfD+TZGOyZPenUAmEOWIiQk3SXpfklXdHNLHiCxNPVhNDIiNgAbei/MEaYDvwX8BvAM8HVJ90XE1zudNJGMyCdOrHxm41B+3si9wMktr+fl20Y9Ju+nzQKeJqsFvxkRT0XEM8AW4I3d3NKYJM1uW+YA90o6XtLsDud5IrL1pvzRyO3AQkkLJB0FrAQG244ZBFbn68uBrRERwF3A6ZKOzYPwbODhogsWNSOfAh5v2zYXuB8I4FWjndRafX9Gr42iQphVLSIOS7qMLHCmATdGxE5J1wA7ImIQuAG4RdIQsI8sIImI/ZI+SxawAWyJiK8WXbMo2D5Klmr8oxHxIICkxyJiwcRu0axLFQzdRcQWsiZg67YrW9afBVaMce6XyIb/u1aUFuFaSZuA9ZL2AJ8gi2SzcjVxulZEDAMrJF0A3A0cW3qpzGr4UKrrW8rbsOcCb4NfZEQ2sy6N6/MjIv49Ih7KXzojspWnad/UdkZkmzQ1bEY6I7KlKcGaqVfOiGxpalqwOSOyWf94IrKlqYF9NrPJUcNmpDMiW5rKn/VfOWdEtjRJvS+JSTD+zerJfTZLU3oVU88cbJamBJuBvXKwWZrqF2vus5lVxTWbpcnNSLOK1LDN5WCzNLlmM6tI/WKtjpW1WZpcs1maatiMLMqI/ApJfy7pFkkXte27vsN5nohsvVEflsQUNSM3khX7K8BKSV+RNCPfd9ZYJ3kisvWsgRORXx0RayPi9oi4gCzt+NY857+ZjUNRn22GpIGIeAEgIj4paS/wTWBm6aWz5qrh0F3RLd0BnNe6ISJuAj4CHCqpTGa1bEYWJfwZ9RcVI+JOSZ8qp0hmJDnA0ateKmtnRLbyNK1mc0Zks/5xRmRLUoIVU8+cEdnSVMNoc0ZkS1P9Ys1zIy1RTtJqZhPljMiWpgZORJ4QT0S2njXtOZvZpEkvVnrmPptZRVyzWZoSbAb2ysFmaapfrDnYLFE1fM7mYLM01S/WPEBiVhXXbJYmD5CYVaR+seZgs0S5ZjOrSP1irTAj8ixJn5b0qKR9kp6W9Ei+bcxJj56IbHakotHI28hSIpwTEbMjYg5wbr7ttrFO8kRk69mAel8KSFoqaZekIUlrR9k/Q9KmfP89kua37X+lpIOS/qirWyrYPz8i1kXEEyMbIuKJiFgHnNLNBcwmpORZ/5KmAdcB7wQWAe+XtKjtsEuA/RFxKrAeWNe2/7PA33d7S0XB9rikKyT9IpOWpJMkfQzY0+1FzMat/K/YnAkMRcTuiDgE3AosaztmGXBzvr4ZWCJlbyzpQuAxYGe3t1QUbO8D5gDfyPts+4BtwGxgRbcXMUvQXF5aYQzn20Y9JiIOAweAOZJmAh9jnLlTixL+7M/f9GPt+yR9gOxXbsz6rw9D/5LWAGtaNm2IiA09vzFcBayPiIMaRzl7Gfq/GgeblUW9zyTMA2us4NoLnNzyel6+bbRjhiVNB2YBTwNvApZL+gvgOOAFSc9GxH/vVB5nRLY0lT/rfzuwUNICsqBaCbSnZxwEVgPfBpYDWyMigLeMHCDpKuBgUaCBMyJbqkqeQRIRhyVdBtwFTANujIidkq4BdkTEIHADcIukIWAfWUBOmDMiW2NFxBZgS9u2K1vWn6VgIDAirur2es6IbGnqQ58tNZ4baWnyRGSzitQwLYIzIluaNND7khhnRDariJuRlib32cwq4mAzq0iCfa5e1e+OzBJVes32fNkXmELmnBCTXYSpo4ZD/25GWprcZzOrSA37bA42S1MNa7b6fXyYJaroy6NHkX2H58cR8TVJFwG/CTxC9hXzn1dQRmuiBg6QbMyPOVbSamAm8DfAErLsRKvLLZ41VgP7bKdHxOvz/At7gV+NiOclfQn43lgntSZaeQ8n8ibPj7TxamCfbSBvSv4ScCxZwhOAGcDLxjqpdSKyA80mpPy8kZUrqtluAB4ly9HwceDLknYDZ5EltTSzLhWlRVgvaVO+/mNJfw28DfhiRNxbRQGtoRKsmXpV+JwtIn7csv4TsjTMZuUaaN4AidnkaGLN1qtny77AFHLCWzxY1GSu2SxNrtnMKtLAh9pmk6OB07XMJkcNm5H1q6vNEuWazdJUwz5b6RmRdzgjsk1EDedGlp4RebEnIttE1DDY3Iy0NNVwulb97sgsUa7ZLFHpNQN75WCzNCXY5+pV6cH2xrIvMIVo4csnuwhTh4f+zWyi3Iy0RLkZaVYN99nMKtK0PpukN0l6Rb5+jKSrJd0haZ2kWZ3ONeuN+rCkpejj40bgmXz9c2R5I9fl2zaWWC6z2ilqRg5ExOF8fXFEjIzk/5OkB8Y6qTUj8n/mRJZ6fqSNVw37bEU120OSPpCvf0/SYgBJrwHG/FGN1onIDjSbmIE+LGkpKtHvA2dL+hdgEfDtPCPyF/N9ZuVo2qz/iDgAXJwPkizIjx+OiCerKJw1WILB0quuhv4j4qd0+NUaMyvm52yWqIbWbL047QTnRP6FGqZnK00NH2q7ZrM01bDPVr+PD7NEOdgsUeVP15K0VNIuSUOS1o6yf4akTfn+eyTNz7e/XdJ9kh7M/z2vmztyM9LSVHKfTdI04Drg7cAwsF3SYEQ83HLYJcD+iDhV0kqyqYrvA54C3p3/QOivAXcBc4uu6ZrNkiSp56XAmcBQROyOiENkP1u9rO2YZcDN+fpmYIkkRcR3W34kdCdwjKQZRRd0sFmiem9GtiYLzpc1LReYC+xpeT3MkbXTL47J5wgfAOa0HfNe4P6IeK7ojkppRrZORL5m5mxWHj2zjMuYdRQRG4ANZb2/pNeRNS3P7+b4UoKt9Sb/+ZdPiTKuYTVX/nO2vcDJLa/n5dtGO2ZY0nSyr5g9DSBpHvC/gFUR8S/dXNDNSEtU6aOR24GFkhZIOgpYCQy2HTMIrM7XlwNbIyIkHQd8FVgbEd/q9o4cbJamkmf9532wy8hGEh8BbouInZKukXRBftgNwBxJQ8B/BUYeD1wGnApcKemBfDmx6JY89G+NFRFbgC1t265sWX8WWDHKeX8G/Nl4r+dgszR5buT4vWrlL5d9iSkjHjk42UWYQuo3N9I1m6WphhORHWyWpho2I+t3R2aJcs1miapfM3LcNZukvy6jIGYv0bTsWpLan6gLODd/gk5EXHDkWWb9UL8eTlEzch7wMPA/gCALtsXAtZ1Oap2I/FfnvJJLf83D/zZOCdZMvSr6+FgM3Ad8HDgQEduAf4+Ib0TEN8Y6qTUjsgPNLFOUpPUFYL2kL+f/Pll0jllf1LBm6zZJ6zCwQtLvAD8tt0hm0MQ+20tExFfJvlpgVq4a1mz1+/gwS1Tp/a+BJf+h7EtMHT89NNklmELqV7N5sMPSVMO5kQ42S1MN+2wONktU/YKtfnW1WaJcs1ma3Gczq4qbkV1pTfu84X//qIxLWN017Ss2E9WaETluf5czItsE1K8ZWb87MkuU+2yWpgSbgb1ysFmaPBppVhXXbOOmc3+v7EtMHT//2WSXwCaRazZLk/tsZlVxn82sGq7ZzKpSv5qt8I4knSZpiaSZbduXllcss/rpGGySLgf+FvgQ8JCkZS27P1Vmwazhajg3sqhmuxQ4IyIuBM4B/kTSh/N9Y97NSyYi3/T1/pTUmqWGwVbUZxuIiIMAEfFDSecAmyWdQodga52IzIFbPRHZJqB5fbYnJb1h5EUeeO8CTgBOL7Ng1nA1rNmKgm0V8ETrhog4HBGrgLeWViqzGirK9T/cYd+3+l8csxHp1Uy98nM2S5Nn/U/ArIWlX8LqqH41W/0+PswS5WakpcnNSLOq1K8Z6WCzNCX4nKxXDjZLUw2bkfW7I7NElZ8RecPflHEJqz31YUmLIsqeJ3yfJyLbKM7oHA0vfLv3/zcDb04q4txns0TVr4dTvzuyeqhg1r+kpZJ2SRqStHaU/TMkbcr33yNpfsu+P86375L0jm5uycFmjSRpGnAd8E5gEfB+SYvaDrsE2B8RpwLrgXX5uYuAlcDrgKXA9fn7deRgs0QN9GHp6ExgKCJ2R8Qh4FZgWdsxy4Cb8/XNwBJJyrffGhHPRcRjwFD+foV3ZJae8puRc4E9La+H822jHhMRh4EDwJwuzz1CBQMkBaNOFZG0Jk/X0HhT42/R+/8bSWuANS2bNkzmfTepZltTfEhjNOJvEREbImJxy9IaaHuBk1tez8u3MdoxkqYDs4Cnuzz3CE0KNrNW24GFkhZIOopswGOw7ZhBYHW+vhzYGtmD6UFgZT5auQBYCNxbdEE/Z7NGiojDki4D7gKmATdGxE5J1wA7ImIQuAG4RdIQsI8sIMmPuw14GDgM/JeIeL7omhXMIEnD1OinVMN/i8nRmGAzm2zus5lVpLbBJunC1hkBkrZJWjyZZSqbpI9L2inp+5IekPSmyS6TvajOAyQXAn9H1ontiaTp+UPNZEl6M1m26jdGxHOSTgCO6vLc5O+vDqZUzSbpdkn35Z/ea/JtB1v2L5d0k6TfBC4A/jL/hH91fsgKSfdK+oGkt+TnHC1po6QHJX1X0rn59oslDUraCkyFXwf5FeCpiHgOICKeiogfS7pS0nZJD0nakE83Gqnp/5ukHcCHJb07n2z7XUlfk3TSZN5MLUXElFmA2fm/xwAPkU2dOdiyfzlwU75+E7C8Zd824Np8/beBr+XrHyEb9gU4DfgRcDRwMdk0nNmTfd9d/m1mAg8APwCuB85u/Zvl67cA7275e1zfsu94Xhww+/2Rv5WX/i1TrRl5uaT35Osnkz1MHI+Rr43fB8zP138L+DxARDwq6XHgNfm+uyNi38SLW52IOCjpDOAtwLnApvxrI/8m6QrgWGA2sBO4Iz9tU8tbzMvP+RWy5udjlRW+IaZMsOU/V/U24M0R8YykbWQ1UOuzi6ML3ua5/N/n6e7efzbOYk6qyB6sbgO2SXoQ+APg9cDiiNgj6Spe+jdqvb/PA5+NiMH8b31VFWVukqnUZ5tF9t2iZySdBpyVb39S0n+UNAC8p+X4fwN+qYv3/UfgdwEkvQZ4JbCrf8WuhqTXSmqt6d/Ai/fxVP4zzcs7vMUsXpzft7rDcTZBU6ZmA+4EPijpEbL/RN/Jt68lG3X8f8AOsr4LZN9P+mL+U8Wd/pNdD/xVXhMcBi6ObDSvhFso1Uzg85KOI7uPIbIJxz8h698+QTYfcCxXAV+WtB/YCiwotbQN5BkkZhWZSs1IsynNwWZWEQebWUUcbGYVcbCZVcTBZlYRB5tZRRxsZhX5/yQyLOEVw4WRAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 216x432 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uTNGmL9A0HNm",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "outputId": "59f6b62a-7f05-46d6-a805-5190c7afc168"
      },
      "source": [
        "res_att_mat = flow_values_gt_2\n",
        "\n",
        "params = {'tokens':tokens, \n",
        "          'attentions': res_att_mat.tolist()\n",
        "          }\n",
        "\n",
        "import json\n",
        "import datetime\n",
        "from google.colab import files\n",
        "time_str = str(int(datetime.datetime.now().strftime(\"%s\")) * 1000 )\n",
        "filename = 'distilbert_attentions_{}.json'.format(time_str)\n",
        "with open(filename, 'w', encoding='utf-8') as f:\n",
        "    json.dump(params, f)\n",
        "\n",
        "files.download(filename)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_f98f92aa-c700-489c-acd4-401d3ccb7281\", \"distilbert_attentions_1606209661000.json\", 51185)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-h2f0PdzzxEg",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 287
        },
        "outputId": "d602edcf-b7f7-45bd-9ce9-b9e13aee58c6"
      },
      "source": [
        "import seaborn as sns\n",
        "sns.heatmap(flow_values_gt_2)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f0c15cce6a0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWYAAAD8CAYAAABErA6HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5hcVZnv8e8vnQu5SYAIhiRKkAQMd4yRMw5yiWDCMMmo6ARG5aYBJYrKyE0PzuD4PCAMiiOHY064OQIRI2hGIpDhqDx6TCRECLkAhhAgIRAuAQwh6XT3e/7Yq0PRdFftrtp71a7q98NTT6p27dprVaisWrXWetcrM8M551xx9Kt3BZxzzr2VN8zOOVcw3jA751zBeMPsnHMF4w2zc84VjDfMzjlXMLk1zJKmSnpM0hpJF+VVjnPONRvlsY5ZUgvwOHA8sB54ADjFzFZlXphzzjWZvHrMk4E1ZrbWzFqBecCMnMpyzrmm0j+n644Gnil5vB74YI+VGDjaww+dc6m0tW5QrdfY8eLa1G3OgJH71lxeb9Vt8k/SLElLJS3t6Hi9XtVwzrnCyavHvAEYW/J4TDi2k5nNAeaA95idc5F1tNe7BmXl1TA/AIyXNI6kQZ4JnJpTWc451zvtbfWuQVm5NMxm1iZpNnAP0ALcYGYr8yjLOed6y6yj3lUoK68eM2a2EFiY1/Wdc65qHX20YXbOucLqqz1m55wrrIJP/tW0XE7SDZI2SVpRcuxKSY9KWi7pTkkjaq+mc85lyDrS3+qg1h7zTcAPgR+XHFsEXBwmAK8ALgYuLHeRo/acWGM10rvrm3HKOuib90cpB2Dztr9GK0uKt9Y+ZtqzDuKUdfiIcVHKAVj68hPRytpryG7RysqCNfOqDDO7X9I+XY7dW/JwMXBypetss3h/Se86/7+ilPP87V+OUg7A35z1s2hl/eW1Z6OV1S/il0CL4sRabdi+OUo5AGOGjYxW1vNb472vTPTxyb8zgZ9294SkWcAsgIkjDmTMsLHdnZa5WP8Ajzjj1ijlACy7Zlq0ssaefVu0sjoi9pgH9GuJUk6szx/AK61bopUV6xdHZvrq5J+kbwBtwC3dPV8a+Tdt7LRo/1d3RBr0f2HbK1HKAdjvi/OjlfX0FR+NVta+F/93tLLaIn0uRvQfGqUcgPVbXoxW1qihu0crKxMFn/zLpWGWdDpwEjDFUgwUbulozaMa3bJI3+wxx2L7R+rtAbzrn++KVtbzPzolWlkHfiXOkvsBivf/KtavAIDXWhtsv5u+1mOWNBW4ADjazLamec2QfgOzrkbdxfzJ2o94XwIxhxdGf+H2aGWtv/LEKOUc/+1HopQDMLAl3mrYXfo32L/hZp78k3QbcAwwUtJ64FskqzAGAYtCr3GxmZ1T7jrD+w2opRq9EqvBfGVbvB7EK8Qra8iAQdHKeqMt3i+pWJPCLz0Vb3hmj/d8JFpZwwYMiVZWJpp58s/MuvuteX1vrxPzryjWGHOz2t6+o95VyEV7pJ+2F066JEo5EPfXzYrrZ0YrKwtmxW4HChH5d2nEsp7Z7T1xytn6QpRyYnvv0FHRylq39floZQ1qifNT/PGOeGvOd99lWLSyjv1CvG1xFs+4oPaLNOsYs6SxJIElewEGzDGza0qePx+4CninmZWdHp6rwdVWo9c+2/KOKOVc1RJvVUZrxPGyNyJO1MYNZonzD3VgxMk/V0aGQxlhXu0akp0055rZ5d2c8yngX0jayofNrOw2yLX0mNuA881smaThwIOSFpnZqtBonwA8neZC7RHXQM7dsTZKOTOHHxilHIA7X/9LtLJiijmpqUhzDzFXIO3SEm8+oD3qgGQGMvoiDomnr6Uk8bSkBaWJpyWNJ5l7+5CZbZa0Z6XrVt0wm9lGYGO4/1dJq0ly/a0CvkeyMuOXaa41IOI/wFjmb3ksWlknD9s/Wln3bV8frayYFOkzGKscgLaI46ixxugzk91cyc7E0wCSOhNPryo55/PAtWa2GcDMNlW6aCZjzCEs+3BgiaQZwAYzezjtT9G2iD3mWB+g7RF7Rre9tqLySRm5YOih0cq6svXVaGXFMkTxpnXaIzbM/RttiKYXQxmlUcrBnBAgB+kST08I1/kDyXDHv5jZ3eXKrPlTImkY8HPgKyTDG5eQDGNUet3ON3vU7kfwvuH71lqVVGLtv7D7wOG8uiPOMraYP/lvaFsXrayYThl2QJRyHu54LUo5AC0RG8uYn8FM9KKDVhqlXKX+wHiSpcVjgPslHWxmPU5E1bqOeQBJo3yLmd0h6WBgHNDZWx4DLJM02cyeK31t6Zv94j6fitZlbomUGPy5bfE2dYkVzQjwzoj/2GP+PL51y+oo5cxtGR+lHIAzYg5lNNoYc3aTfxUTT5P0opeY2Q7gSUmPkzTUD/R00VpWZYhkzfJqM7sawMweAfYsOWcdMKnSqozXifcB2hHpwzpiYLylSlvbt0Ura3tHvHXMMXthsQKPrhwYb7ncZwYfFK2s+3fEW9qYiewa5jSJp38BnALcKGkkydBG2VUItfSYPwR8BnhE0kPh2CUh11+v7E68yL9YNrfG+wcYc9/ifgPiNZYxg4GkOF84Y1rifWHf9NryaGX9zyGHRSsrC5bR5F9PiaclXQYsNbMF4bkTJK0C2oGvm9lL5a5by6qM30P5Lo2Z7ZPmWqM74k2IDI4USDC80UJUUxoUMXx+14HxdmKL1WPe1PFGlHIAhveP9xm8ybr+es/P2VlcJMNhsu4ST5vZpSX3DfhauKVSiMi/Df3iBUjsNyDO9oQP7ki1f1MmYi6Lihlgsr09XlkDIn3hvLPfLlHKAVjREe/fVcxfbZlo5r0ysrIt4sTBI9ueq3xSBg4dMjpKOQCrt8Ub34s50Rhz8/VYq3W2RvwSjbnDYcwozUwUfN11LZN/uwD3k+wk1x+Yb2bfCpOC/wZ8kmQ85Toz+0G5aw2MtFIC4s30P7w13k+7mCYPeXe0sv6wI17OulgGRWwsY36Jeo85W7X0mLcDx5nZlrBs7veSfg28j2T5yAFm1pEm/DDqByhSWTHfU8xosuXbN0Yr6/BhcTacAlj9RpxfUi1Nune2p5bKVi2TfwZ0JhUbEG4GfAE41cKuMGnCD3c0YSMW9WdkzD0lIpb14JZ10cqaPCxO9uqYn/WYn8GYmVky0dbcG+W3AA8C+5HEgi+R9F7gHyV9DHgB+LKZvW2XndLIv2N2fz8HDn9vLVVJLdaqjFcjTv7FNKx/vMmrQREz26x4I84vgVMHxwsw+WO0kuLFB2SmWXvMAJbsNn2YpBHAnZIOIhlz3mZmkyR9HLgBOKqb174l8i/WT6Gt7dujlDO0f7ydvbZF3Lw+ZoBJzP1GBkfaie1x4i2XO2TI3tHK2tgWLyN3Jpp4jHknM3tF0m+AqSThh3eEp+4EbsyijKzE2mwlZmMZczzb1WZIxJ/8i19/KlpZxw2Ns9dNZpq1xyzpncCO0CgPJtmP9AqS8MNjgSeBo4HHK12rNeJyuWEtcX6Kb48USQZxJ14GRxxe2KJ4oeaxPhcbOuINccV6TwCrd7wcraxMNHGPeRRwcxhn7gfcbma/kvR74BZJXyWZHPxcpQu9bPF+ssYKxhgzcLco5QBs2BEvW0rMoYy2iAES2yK9r5gZTFot3t/fsIib8meiWXvMZracZA/mrsdfAf6ulkrlKVbvcuOOeHsJ79V/eLSyXmyPl5Hb1SbmcrmGW8fczKsyGlHD7RubwvNtERN89o+3f8WLivflFivyzxVEwb9IstgovwVYSpK15CRJU4ArSYY3tgCnm9macteI+W0bK/IvZohqzH2LX9oRb/Z97C4jo5X1anuc1RIxP+s+KVxGE48xdzoPWA10pp++DphhZqslfRH4JnB6uQvEbMRilRWzZ25Rk5bGK2vdGxVjkzIzfsioKOXE/PuLGWASs6xMNHPDLGkMyXjyd3hzSzvjzUZ6V+DZStcZGnFEJdYHqC3iXsIxNes/9pfb4oyd7x1xP+aBEfMLvhYxWUMmmnXyL/g+STbs0tmnzwELJb0BvAYcWeki2yJmMIk5lBGrYYk5lBGzrJjLAGOlRor5WY8ZjbdX/3hfOJloL3bHqZZ1zCcBm8zsQUnHlDz1VeDEEJ79deBqulkyVxqSfeBuB/LuYXF2LYu50XssMfeviDlEE7OsWIFHr1u85Yaxth8A2NzRYD3mJh7K+BAwXdKJwC7AOyTdRbKr3JJwzk+BbtN0l4ZkTxs7zWJNisQqJ+amLg2XCLMPizr5F7GsARG37s1EszbMZnYxcDFA6DH/M/APwHOSJpjZ4yTRgBXTDxf7r6g6g/rFG9/b3h5vTWbDbe+YUqxGrBk/69CA76vJx5jfIiQm/Dzwc0kdwGbgzCzLaBSvR9yAZ0jE4ZntjbaLWB/WrF+iWbCOYv/dZLWJ0W+B34b7d5JsXuQiifklEHOmX54ayeWlWYcystSMeeT6RfxCjjmW2Eq8YZM9BsYLNY+12qRZ0z01XDBLs67KAAj7MM8FDiJZv3wm8BjJpN8+wDrgU2a2udx1Ym7sEmtSLuaysqgBOhFXSrywPd7mTCMGxFnuFfOz3qyfi0w0eY/5GuBuMztZ0kBgCHAJcJ+ZXS7pIuAi4MJyF9nRhOtwo/40brDOSloxx0hj/f+K+VmPuUVrw8mwYZY0laQtbAHmmtnlXZ4/nWSbis4MzT80s7nlrlnLOuZdgQ8Twq3NrBVolTQDOCacdjPJ2HPZhjnmQhsfS2wczbjhVMzPeswtWt/Rr9G2/czmSz/sFXQtyQq09cADkhaY2aoup/7UzGanvW4tPeZxJDn9bpR0KEnuv/OAvcysM4Hac8BelS7UGnGmf0ekPX5j7k8bswGLGU02IOJqk1gBJjE/6/0iTp7GfF+ZyK7HPBlYY2ZrASTNA2YAXRvmXqnl/1x/4AjgOjM7HHidZNhipxA10u1Xk6RZkpZKWrrh9fU1VMM553qpw9LfyhsNPFPyeH041tUnJC2XNF/S2EoXraVhXg+sL4nym0/SUD8vaRRA+LPbLcLMbI6ZTTKzSaOHjqmhGs4510vt7alvpZ3IcJvVy9L+C9jHzA4BFpEM8ZZVS+Tfc5KekbS/mT0GTCHpvq8CTgMuD3/+stoynHMuD9aLoYzS7SO6sQEo7QGP4c1Jvs7Xv1TycC7w3Upl1roq40sk+f0GAmuBMwj5/ySdBTwFfKrSRZoxQilmWp9mNax/vGSisdbhxvysN2N8QGayi/x7ABgvaRxJgzwTOLX0BEmjSubdppNim4qaGmYzewiY1M1TU3pznaacfY+awaTB/lGktKUt3o5lIwbESZkV87PerLsOZiKjZYthG4rZwD0ky+VuMLOVki4DlprZAuDLkqYDbcDLVEgcAgWJ/IvJl8s1joaLJnONI8O9MsxsIbCwy7FLS+7v3PAtrT7XMDdcNt8+rOGiyVzjaCv28r5aAkz2Jwm97rQvcCnJUpG/B1qBJ4AzzCxebK1zzlXSrNt+hpUYh8HO6JcNJLvK7Q9cHMZeriDpwpeN/Gu4iYMUfPKvdjEzcPjkX20a7t9wX9j2k2Sy7wkze4pkJUanxcDJGZXRULZ2tDLE9yqoScNNKKWwpWM7wxotfLkJ9Wa5XD1k1TDPBG7r5viZvHW4Y6fSnH8TRhzA3k0WZOKNcu0arheWQsxG2cfoyyh4j7nmYPqwhnk68LMux79Bsjzklu5eVxr512yNsnOu4LILyc5FFj3macAyM3u+80DY5u4kYEq0LKuu6bRE3ISnGflywzKaeaP84BRKhjHC3qQXAEeb2dYMru/6qKjf6f6rv09p6px/koaS7EN6dsnhHwKDgEUhmGOxmZ1TSzmub2qPmHt5QBMu6fcx5jKauWE2s9eBPboc26+mGjkXxGxY/Gd/H9NHVmU4lzkfY66Nf9mUUfAec02ffElflbRS0gpJt0napeS5H0jaUnsVnXMuYwVflVF1wyxpNPBlYJKZHUSys9LM8NwkYLdMauiccxmz9o7Ut3qo9bdif2CwpP4kGbKfDeHZV5KszHDOueJp1h6zmW0ArgKeBjYCr5rZvcBsYEHJxtDOOVco1mGpb/VQy1DGbiTZYMcBewNDJX0W+CTwHylevzOP1rOejNU5F1Oz9piBjwBPmtkLZrYDuAP4V2A/YI2kdcAQSWu6e7GHZDvn6qajF7c6qGW53NPAkZKGAG+Q7DB3tZnt7C1L2uLrmp1zRWNtTbqO2cyWSJoPLCPZrOjP9JxJ1jnniqPY7XLNkX/fAr5V5vlhtVzfOefy0NR7ZTjnXENq5h6zc841oqL3mGsNyT4vhGOvlPSVkuNfkvRoOP7d2qvpnHMZatZVGZIOAj4PTCbJiH23pF8BY0nWNx9qZtsl7ZlJTRtMzJx/LYh237CmIXjOv2KwtnrXoLxaeszvA5aY2VYzawN+B3wc+AJwuZltBzCzTbVXs/HEzPnnjXLj8Ea5GKwj/a0SSVMlPSZpjaSLypz3CUkW9hIqq5aGeQVwlKQ9wlrmE0l6yxPC8SWSfifpAz1U0iP/nHP1kdFQRtgb6FqSFHsTgVMkTezmvOHAecCSNNWrZa+M1cAVwL3A3cBDQDvJ8MjuwJHA14HbFVKZdHm9R/455+oiwx7zZGCNma01s1ZgHslQblffJmkvt6WpX02Tf2Z2vZm938w+DGwGHgfWA3dY4k8k3zkjaynHOeeylGHDPBp4puTx+nBsJ0lHAGPN7K609as159+eZrZJ0rtJxpePJGmIjwV+I2kCMBB4sZZynHMuS9aePm2ZpFnArJJDc8wsVZSzpH7A1cDpvalfreuYfy5pD2AHcK6ZvSLpBuAGSStIVmucZlHTHTvnXHlpJvV2nps0wj01xBtI5tY6jQnHOg0HDgJ+G0Z03wUskDTdzJb2VGatIdlHdXOsFfh0Ldd1zrk8WUdmiX4fAMZLGkfSIM8ETt1ZjtmrlAzlSvot8M/lGmXwyD/nXB/Umx5z2euYtUmaDdxDkl7vBjNbKekyYKmZLajmuqka5jA8cRKwKeT3Q9LuwE+BfYB1wKfMbLOkXYGfAO8O17/KzG6spnLOOZcHs8x6zJjZQmBhl2OX9nDuMWmumXZVxk3A1C7HLgLuM7PxwH3hMcC5wCozOxQ4Bvh3SfGiLZxzroIsA0zykKphNrP7gZe7HJ4B3Bzu3wz8Q+fpwPCwdnlYeF3BAyCdc31JR7tS3+qhljHmvUoSrj4H7BXu/xBYADxLMiP5j2b1+t5xzrm3y3DyLxc1BZh0CsvhOpfEfZQkCnBv4DDgh5Le0fU1HpLtnKsX61DqWz3U0jA/L2kUQPizc7OiM3gz8m8N8CRwQNcXe0i2c65ezNLf6qGWhnkBcFq4fxrwy3D/aZLErEjaC9gfWFtDOc45l6mi95jTLpe7jWSFxUhJ60ny/F1OskHRWcBTwKfC6d8GbpL0CCDgQjPzkGznXGFkuVwuD6kaZjM7pYenpnRz7rPACbVUyjnn8tRep9UWaXnkn3Ouz2mKHrNzzjWThl8uJ+kGSZvCbnGdx64MyVaXS7pT0oiS5y4OKVYek/TRvCpedFs7WqOV1UKxP2TuTVs6tkcry9NY9awZVmXcxNvDsRcBB5nZISSb418MEFKqzAQODK/5XyH1Sp/jOf9cd2I2ljG/BBpN0VdlVGyYuwvHNrN7QwJWgMUke5BCEqY9z8y2m9mTwBqS1CvOOVcY7R39Ut/qIYtSzwR+He5XTLPSySP/nHP10gxDGT2S9A2SDYpu6e1rPfLPOVcvHabUt3qoelWGpNNJ9mieUpI6qlKaFeecq7uiL5erqscsaSpwATDdzLaWPLUAmClpUEi1Mh74U+3VdM657BR9KKNij7mHcOyLgUHAopBgcLGZnRNSqtwOrCIZ4jjXzNrzqrxzzlWjXkMUaVVsmHsIx76+zPnfAb5TS6Wccy5P9VptkZZH/jnn+pyir/yvNvLv2yHq7yFJ90raOxz/p3D8EUn/T9KheVbeOeeqUfRVGdVG/l1pZoeY2WHAr4DOjLBPAkeb2cEk23/OyaqizjmXFTOlvtVDmjHm+yXt0+XYayUPhxJ+GZjZ/ys5XhoR6JxzhVH0JKS1rGP+DvBZ4FXg2G5OOYs3IwKdc64wrOAbf1U9NWlm3zCzsSRRf7NLn5N0LEnDfGFPr/eQbOdcvbSZUt8qkTQ17Ka5RtJF3Tx/Tph3e0jS78Nmb2VlsWbkFuATJZU4BJgLzDCzl3p6kYdkO+fqxVDqWzlh98xrgWnAROCUbhreW83s4DAn913g6kr1qzbyb3zJwxnAo+H4u4E7gM+Y2ePVXNs55/LW0YtbBZOBNWa21sxagXkkbeJOPc3JlVNt5N+JkvYP9X4KOCecfimwB8k+zABtZjapUhnOORdThmPM3e2o+cGuJ0k6F/gaMBA4rtJFM438M7PPAZ+rdE3nnKun3qzKkDQLmFVyaI6Z9WopsJldC1wr6VTgm8Bp5c73yD/nXJ/T3osec2iEe2qIe7uj5jzgukplphpj7i76r+S58yWZpJFdjn9AUpukk9OU4ZxzsXQo/a2CB4DxksZJGkiSWm9B6Qld5uT+DvhLpYumnfy7ibdH/yFpLHAC8HSX4y3AFcC9Ka/vnHPRdKDUt3JCir3ZwD3AauD2sMvmZZKmh9NmS1op6SGSceaywxiQciiju+i/4Hsk+zL/ssvxLwE/Bz6Q5vrOORdTlpsYmdlCYGGXY5eW3D+vt9esJfJvBrDBzB4OKzA6j48GPkYSDegNs3OucJoyJFvSEOASkmGMrr4PXGhmHaUNdjfX2DnTOWHEAXiQiXMulo4ybVMRVNtjfi8wDujsLY8BlkmaDEwC5oXjI0nWPLeZ2S9KL1A603nsmOOLvj2qc66JFD2tUlUNs5k9AuzZ+VjSOmCSmb1I0mB3Hr8J+FXXRtk55+opxWqLukq7XO424I/A/pLWSzor32o551x+slqVkZe0qzK6i/4rfX6fHo6f3vsqOedcvoo+duqRf865PqfoQxneMDvn+pyiL5erNhnrv0jaEDZ+fkjSiSXPHSLpjyHS5RFJu+RVeeecq0a70t/qodpkrADfM7PDwm0hgKT+wE+Ac8zsQJLtQndkVFfnnMtEhvsx56KqZKxlnAAsN7OHw2t7zGDinHP10vBDGWXMlrQ8DHXsFo5NAEzSPZKWSbqgpxd7zj/nXL2Y0t/qodqG+TqS6L/DgI3Av4fj/YG/Bf4p/PkxSVO6u4Dn/HPO1UvRhzKqapjN7HkzazezDuD/kOS9giStyv1m9qKZbSXZcemIbKrqnHPZaO/FrR6qTcY6quThx4DOFRv3AAdLGhImAo8GVtVWReecy1aGG+XnotpkrMdIOowkgGYdcDaAmW2WdDXJrv4GLDSzu/KpunPOVafok3+ZJmMN5/+EZMmcc84VUsM3zM4512yKvldGVZF/4fiXJD0aIvy+G44NkHRziPhbLenivCrunHPVavgxZpLIvx8CP+48IOlYYAZwqJltl9S5N/MngUFmdnDIcrJK0m1mti7bajvnXPUafqP8HiL/vgBcbmbbwzmbOk8HhoYVGYOBVuC1zGrrnHMZ6Cj4YEa1ASYTgKMkLZH0O0mdSVfnA6+TBJ08DVxlZi9nUE/nnMtMUwaYkPS0dweOBL4O3K4kyd9kkl8Je5OkmDpf0r7dXcBDsp1z9WK9uNVDtQ3zeuAOS/yJ5ItlJHAqcLeZ7QjDG38gSc76Nh6S7Zyrl2btMf8COBZA0gRgIPAiyfDFceH4UJIe9aO1V9M557LTJkt9q0TSVEmPSVoj6aJunv+apFVh07f7JL2n0jXTLJfrLhHrDcC+YQndPOA0MzPgWmCYpJUk0X83mtnyiu/MOeciymooQ1ILSbs3DZgInCJpYpfT/gxMMrNDSObhvlupftVG/gF8uptzt5AsmXPOucLKcIhiMrDGzNYCSJpHspR45x5BZvabkvMX003b2VUt+zE751xD6sBS30oXKoTbrJJLjQaeKXm8PhzryVnAryvVz0OynXN9Tm9WW5jZHGBOrWVK+jTJYoijK52bqsfcQ0LWwyQtDslYl0qaHI5L0g/CQPhySb4fs3OuUDJclbEBGFvyeEw49haSPgJ8A5jeGZhXTtqhjJt4e0LW7wL/amaHAZfy5oD2NGB8uM0iyXbinHOF0Y6lvlXwADBe0jhJA4GZwILSEyQdDvyIpFHe1M013iZVw2xm9wNdI/gMeEe4vyvwbLg/A/hxWOO8GBjRZWN955yrq6x6zGbWBswmSRKyGrjdzFZKukzS9HDalcAw4GdhhGFBD5fbqZYx5q8A90i6iqSB/5twvKfB8I2lLw4D6LMAJow4AA8ycc7FYhnG9JnZQpI0eqXHLi25/5HeXrOWVRlfAL5qZmOBr1Jm8/zueOSfc65emjXyD+A04I5w/2e8mZA11WC4c87VS2+Wy9VDLQ3zs7y57OM44C/h/gLgs2F1xpHAq2a2sbsLOOdcPRR9E6NUY8w9JGT9PHBN2Ht5G2G8mGSs5URgDbAVOCPjOjvnXE3aCr4fc6qGuUxY9vu7OdeAc2uplHPO5SnLyb88eOSfc67PKXqW7KqSsUo6VNIfQ9LV/5L0jnD8eEkPhuMPSjouz8o751w1rBf/1UOayb+beHvU31zgIjM7GLiTJIsJJHsy/304fhrwnxnV0znnMtPwy+V6iPqbANwf7i8CPhHO/bOZdUYArgQGSxqUUV2dcy4T7Wapb/VQ7XK5lSSh15Dsvzy2m3M+ASxLs2GHc87F1KzrmM8EvijpQWA40Fr6pKQDgSuAs3u6gCdjdc7VSzOMMb+NmT1qZieY2fuB24AnOp+TNIZk3PmzZvZEmWs0dUj21o7Wyic55+qi6GPMVS2Xk7SnmW2S1A/4JvC/w/ERwF0kE4N/yK6ajWdIv4H1roJzrgf1GqJIq9pkrKdIepwkA/azwI3h9NnAfsClYXu7hyTtmVPdnXOuKkUfyqglGes13Zz7b8C/1Vop55zLU71WW6TlkX/OuT6n6EMZ3jA75/qcZgjJHivpN5JWSVop6bxwfHdJiyT9Jfy5W5fXfUBSm6ST86q8c85Vo+hjzGmWy7UB55vZROBI4FxJE4GLgNWaDFgAAApUSURBVPvMbDxwX3gMgKQWknXM92ZfZeecq03DB5iY2UYzWxbu/5Uk4eBoksi/m8NpNwP/UPKyLwE/B1JlhHXOuZjMLPWtHnoVYCJpH+BwYAmwV0lmkueAvcI5o4GPAddVuJZH/jnn6qIdS32rh9QNs6RhJL3gr5jZa6XPhc3xO9/B94ELzazs+HqzR/4554qr6EMZaVNLDSBplG8xs84ErM9LGmVmGyWN4s1hi0nAPEkAI4ETJbWZ2S8yrrtzzlWlXkMUaaVZlSHgemC1mV1d8tQCkj2XCX/+EsDMxpnZPma2DzAf+KI3ys65IsmyxyxpqqTHJK2RdFE3z39Y0rLerFJLM5TxIeAzwHElYdYnApcDx0v6C/CR8Ng55wovq+VyYQXatcA0YCLJdhUTu5z2NHA6cGva+qUJyf49oB6enlLhtaenrYhzzsWSYUj2ZGCNma0FkDSPZMXaqs4TzGxdeC51XItH/jnn+pwMJ/VGA8+UPF4PfLDWi9YS+ffJ8LhD0qQurzkkJGtdGRKz7lJrRZ1zLiu9GWMuXdobbrPyrl+aHnNn5N8yScOBByUtAlYAHwd+VHqypP7AT4DPmNnDkvYAdmRcb+ecq1pvVmWY2RxgTg9Pb+CtqfXGhGM1STPGvBHYGO7/VdJqYLSZLQIIy+JKnQAsN7OHw2teqrWSzjmXpQyHMh4AxksaR9IgzwROrfWitUT+9WQCYJLuCUtELqi+es45l72sVmWYWRtJgpB7SLaruN3MVkq6TNJ02Lmh23qSxNU/krSyUv1ST/6Vi/zr5pp/C3wA2ArcJ+lBM7uvy/VmAbMAJow4gGaL/tva0erppZwrqPbygcm9YmYLgYVdjl1acv8BkiGO1FL1mHuI/OvJeuB+M3vRzLaGCh/R9aRmD8n2Rtm54mr4TYzKRP715B7gYElDwkTg0ZSs6XPOuXprhr0yOiP/HpH0UDh2CTAI+A/gncBdkh4ys4+a2WZJV5MMihuw0MzuyqHuzjlXlXptgJ9WrZF/d/bwmp+QLJlzzrnC6Sj4JkYe+eec63MavsfsnHPNJstVGXmoJST7SkmPSlou6U5JI8LxAZJuDqHYqyVdnPebcM653ugwS32rh1qSsS4CDjKzQ4DHgc4G+JPAIDM7GHg/cHYITHHOuUIoepbsWkKySzNgLwY6N4A2YGhYKjcYaAXKBaQ451xURZ/8yyok+0zg1+H+fOB1ksb8aeAqM3u5m2t5MlbnXF0UvcdcczJWSd8gGe64JRyaDLQDewPjgPMl7dv1es0e+eecK652a099q4dakrEi6XTgJGCKvRm7eCpwt5ntADZJ+gNJgta1WVbcOeeq1bTJWCVNBS4Apoc9MTo9DRwXzhlKMmH4aJaVds65WjRzSPYPSMKyF4U9mReb2TkkiQlvDFvbCbjRzJZnXnPnnKtS0XvMtYRkL+zmGGa2hWTJnHPOFVLRV2V45J9zrs8pekh21ZF/Jc+fL8kkjQyPJekHktaEqMC37cXsnHP11G4dqW/1UHUyVjNbJWksSY6/p0vOnwaMD7cPAteRQTpv55zLStHHmCv2mM1so5ktC/f/SpLXanR4+nskKzNK3+UM4MeWWAyMkDQq22o751z1mmGvjJ1KI/8kzQA2dGbDLjEaeKbk8XrebMj7jK0drfWugnOuBw2fWqpTaeQfyfDGJcClZV9U/npNHZLtOf+cK66ir2OuNhnre0nCrR+WtI4kA+wySe8CNgBjS14+Jhx7Cw/Jds7VS9F7zBUn/7qL/DOzR4A9S85ZB0wysxclLQBmS5pHMun3atihzjnnCqHoG+VXHflnZt0GmJAEnpwIrAG2AmfUXEvnnMtQwweYVEjG2nnOPiX3DTi35po551xOshyiCPsGXQO0AHPN7PIuzw8CfkySOOQl4B/NbF25a/ZqVYZzzjWDrPZjltRCsj/QNGAicErI8FTqLGCzme1HssT4ikr184bZOdfnZDj5NxlYY2ZrzawVmEcSy1FqBnBzuD8fmBLm7nrkDbNzrs/JMMAkTdzGznPMrA14Fdij3EULsYnRb9YvKvvt0RNJs8xsTtb1qWdZzfiemrWsZnxPzVxWqbbWDanbHEmzgFklh+bkXedG7zHPqnxKw5XVjO+pWctqxvfUzGVVpTTmItxKG+U0cRs7zwlJqnclmQTsUaM3zM45V08PAOMljZM0EJgJLOhyzgLgtHD/ZOD/WoXB60IMZTjnXCMyszZJs4F7SJbL3WBmKyVdBiw1swUkAXr/KWkN8DJJ411WozfMMcemYpXVjO+pWctqxvfUzGXlIgTbLexy7NKS+9voZVYnFX1fUuec62t8jNk55wqmIRtmSVMlPRbSV12UYzk3SNokaUVeZZSUVTaFV8Zl7SLpT5IeDmX9a15lhfJaJP1Z0q9yLmedpEckPSRpac5ljZA0X9KjklZL+h85lbN/eD+dt9ckfSWnsr4aPg8rJN0maZc8ygllnRfKWZnX+2lovYmAKcKNZID9CWBfYCDwMDAxp7I+DBwBrIjwvkYBR4T7w4HHc3xfAoaF+wOAJcCROb63rwG3Ar/K+e9wHTAy7/9Xoaybgc+F+wOBERHKbAGeA96Tw7VHA08Cg8Pj24HTc3ofBwErgCEk81z/DewX4/9bo9wascecJgQyE2Z2P8ksau6sfAqvrMsyM9sSHg4It1wmGySNAf4OmJvH9etB0q4kX9rXA5hZq5m9EqHoKcATZvZUTtfvDwwOa22HAM/mVM77gCVmttWSSLjfAR/PqayG1IgNc9OnripN4ZVjGS1hG9dNwCIzy6us75PkhYyxAa4B90p6MERr5WUc8AJwYxiimStpaI7ldZoJ3JbHhc1sA3AVSWLljST7qN+bR1kkveWjJO0haQjJNsFjK7ymT2nEhrmplabwMrPX8irHzNrN7DCSSKXJkg7KugxJJwGbzOzBrK/dg781syNIdvo6V9KHcyqnP8kQ13VmdjjwOpDbXAdACF6YDvwsp+vvRvLLcxywNzBU0qfzKMvMVpPssHYvcDfwENCeR1mNqhEb5lSpqxpRNym8chd+gv8GmJrD5T8ETA8ZbuYBx0n6SQ7lADt7fZjZJuBOkmGvPKwH1pf8yphP0lDnaRqwzMyez+n6HwGeNLMXzGwHcAfwNzmVhZldb2bvN7MPA5tJ5lRc0IgNc5oQyIbTXQqvHMt6p6QR4f5g4Hjg0azLMbOLzWyMJYkUZpKEoubSC5M0VNLwzvvACSQ/mTNnZs8Bz0jaPxyaAqzKo6wSp5DTMEbwNHCkpCHhsziFZJ4jF5L2DH++m2R8+da8ympEDRf5Zz2EQOZRlqTbgGOAkZLWA98ys+vzKIvep/CqxSjg5rDJdz/gdjPLdSlbBHsBd4ZtbvsDt5rZ3TmW9yXgltA5WEuOKdTCF83xwNl5lWFmSyTNB5YBbcCfyTcq7+eS9gB2AOdGmjxtGB7555xzBdOIQxnOOdfUvGF2zrmC8YbZOecKxhtm55wrGG+YnXOuYLxhds65gvGG2TnnCsYbZuecK5j/D3/clSKpAsAtAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mgD9qqCZzUaM"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}